/*
 * Copyright (c) 2011-2012 Alexander Dubu
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * o  Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * o  Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * o  Neither the name Axil nor the names of its contributors may be used to
 *    endorse or promote products derived from this software without specific
 *    prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package axil.compiler.lexer;

import axil.api.extend.Source;
import axil.compiler.lexer.Literal.Species;
import axil.compiler.lexer.Token.Type;
import axil.definition.OperatorDefinition;
import axil.engine.Config;
import axil.etc.NameValue;
import axil.framework.error.CompilationException;
import axil.stdlib.core.type.Bool;
import axil.stdlib.core.type.Nil;
import axil.stdlib.money.type.Currency;
import axil.stdlib.numbers.type.Fraction;
import axil.stdlib.numbers.type.Int;
import axil.stdlib.numbers.type.Percent;
import axil.stdlib.time.type.Date;
import axil.stdlib.time.type.Time;

import java.io.IOException;
import java.io.Reader;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;

import static axil.framework.Functions.*;


/**
 * The lexical analyzer for the Axil. Since the syntax is designed to be human
 * readable as much as possible, and support concept like application-defined
 * extended literals, writing a traditional LEX/ANTLR grammar would would be
 * extremely difficult. So, this lexer is from the old school, written entirely
 * by hand. Not a piece of code for the faint-of-heart. Make sure you REALLY
 * understand the Axil grammar and have extensive unit tests in place before
 * you start "improving" this.
 */
public class Lexer implements Source {
	private static final Locale LOC = Locale.US;
	private static final Integer ZERO = Integer.valueOf(0);

	private static final char
	STRING = '"',
	CHAR = '\'',
	IDENT = '`',
	OPEN = '(',
	CLOSE = ')',
	COMMA = ',',
	DASH = '-',
	ESCAPE = '\\',
	PERIOD = '.',
	COLON = ':',
	PERCENT = '%',
	COMMENT = ';',
	EOL = '\u0000';

	private static final String operators = "+-*/<>=^&?!";
	private static final String currencies = Currency.symbols();
	private static final Map<Character,Character> alt =
	    new HashMap<Character, Character>();
	static {
		alt.put('\u003C', '<');
		alt.put('\u003E', '>');
		alt.put('\u00F7', '/');
		alt.put('\u00D7', '*');
		alt.put('\u2212', '-');
		alt.put('\u2215', '/');
		alt.put('\u22C5', '*');
		alt.put('\u00B2', '2');
		alt.put('\u00B2', '2');
		alt.put('\u00B3', '3');
		alt.put('\u00B3', '3');
	}
	private static final Map<Character,String> expand =
	    new HashMap<Character, String>();
	static {
		// TODO: Can this be eliminated now that we have operator aliases? See LexerTest, which may be able to verify this.
		expand.put('\u2264', "<=");
		expand.put('\u2265', ">=");
		expand.put('\u2260', "<>");		// Not equal
		expand.put('\u2261', "==");    	// Identical to
		expand.put('\u2263', "=="); 	// Identical to
		expand.put('\u2262', "=/="); 	// Not identical to
		expand.put('\u224D', "=?="); 	// Equivalent to
		expand.put('\u2245', "=?="); 	// Approximately equal
	}
	private static final Map<String,Character> charwords = new HashMap<String, Character>();
	static {
		charwords.put("space", ' ');
		charwords.put("nul", '\u0000');
		charwords.put("ack", '\u0006');
		charwords.put("bell", '\u0007');
		charwords.put("tab", '\u0009');
		charwords.put("lf", (char)0x0A);
		charwords.put("nl", (char)0x0A);
		charwords.put("newline", (char)0x0A);
		charwords.put("ff", '\u000C');
		charwords.put("cr", (char)0x0D);
		charwords.put("return", (char)0x0D);
		charwords.put("esc", '\u001B');
	}
	private static final Map<String,Keyword> keywords = new HashMap<String, Keyword>();
	static {
		keywords.put("true", new Keyword(Bool.TRUE));
		keywords.put("false", new Keyword(Bool.FALSE));
		keywords.put("nil", new Keyword(Nil.object));
	}

	private Config config;
	private Reader input;
	private int source;
	private int line;
	private StringBuilder buffer;
	private boolean as;
	private Pushback pushed;


	public Lexer(Config config, Reader input, int source) {
		assert config != null;
		assert input != null;

		this.config = config;
		this.input = input;
		this.source = source;
		this.line = 1;
		this.as = false;
		this.buffer = null;
		this.pushed = null;

		/*
		 * We allow for a Unix-style shell execution line at the top of a
		 * script. For example: #!/usr/bin/axil. We do not make any attempt to
		 * interpret such a line.
		 */
		char ch = read();
		if (ch == '#') {
			discard();
		} else {
			pushback(ch);
		}
	}


	/**
	 * Get the unique ID assigned to the source element, such as the module,
	 * function or script.  If the object lives outside of source code, then
	 * zero is returned.
	 */
	public int source() {
		return source;
	}


	/**
	 * Get the line number in the original source where this object resides.
	 * If the object exists outside of source code (such as the boolean keyword
	 * "true"), then zero is returned.
	 */
	public int line() {
		return line;
	}


	/**
	 * Get the next token from the lexical analyzer. A null is returned if the
	 * end of the input is reached. An exception is thrown if there are any
	 * gross syntactic errors in the source.
	 */
	public Token next() {
		buffer = new StringBuilder();
		while(true) {
			skip();
			char ch = read();
			switch (ch) {
			case OPEN:
				return new Special(this, Type.open);

			case CLOSE:
				return new Special(this, Type.close);

			case COMMA:
				if (! white(read())) {
					/**
					 * This is necessary to differentiate between something
					 * like this:
					 * 		f(3,250, y)
					 *
					 * Not being incorrectly recognized as this:
					 * 		f(3, 250, y)
					 *
					 * To avoid confusing the user, we simply always require
					 * the whitespace. This allows the advanced parsing
					 * features and also enforces the correct code style.
					 */
					throw problem("comma-whitespace");
				}
				return new Special(this, Type.comma);

			case STRING:
				return string();

			case CHAR:
				return character();

			case IDENT:
				put(ch);
				return escaped();

			case DASH:
				return dash();

			case '0':
				return zero();

			case COMMENT:
				discard();
				break;

			case EOL:
				return new Special(this, Type.EOF);

			default:
				if (as) {
					return as();
				}
				if (digit(ch)) {
					put(ch);
					return number();
				}
				if (letter(ch)) {
					put(ch);
					return resolve();
				}
				if (in(operators, ch)) {
					put(ch);
					return operator();
				}
				if (in(currencies, ch)) {
					if (config.monies()) {
						return money(ch);
					}
					throw syntax("syntax-near", ch);
				}
				String expansion = expand.get(ch);
				if (expansion != null) {
					put(expansion);
					return operator();
				}
				throw syntax("syntax-near", ch);
			}
		}
	}


	/**
	 * This special parsing mode is to support the "as" operator. The "as"
	 * operator supports conversion to unitized types. For example:
	 *
	 * 		velocity as km/s
	 *
	 * This could be done with an as() function, but this conversion fits so
	 * well in the syntax that it is integrated into the language itself for
	 * clarity and expressiveness.
	 */
	private Token as() {
		as = false;
		while(true) {
			char ch = read();
			if (letter(ch) || digit(ch) || (ch == '/')) {
				put(ch);
			} else {
				String b = buf();
				if (b.length() == 0) {
					throw syntax("expecting-units", ch);
				}
				pushback(ch);
				return new Units(this, b);
			}
		}
	}


	/**
	 * Parse out a string literal. String can cross line boundaries, and the
	 * whitespace between the end of lines is compressed into a single space
	 * character. A string can also contain escape sequence in the form of:
	 *
	 * 	Classic:	\n
	 * 	Unicode:	\U+F302
	 *
	 * Keyword escapes are not supported at this time for string.
	 */
	private Token string() {
		int started = line();
		while(true) {
			char ch = read();
			switch(ch) {
			case STRING:
				return new Literal(this, Species.STRING, buf());

			case '\r':
				trim();
				seek();
				put(' ');
				break;

			case '\t':
				throw problem("embedded-tab");

			case ESCAPE:
				ch = read();
				switch(ch) {
				case 'U':
					expect('+');
					StringBuilder b = new StringBuilder(4);
					for (int i = 0; i < 4; i ++) {
						b.append(hex());
					}
					put((char)Integer.parseInt(b.toString(), 16));
					break;

				case 'r':
					put('\r');
					break;

				case 'n':
					put('\n');
					break;

				case 't':
					put('\t');
					break;

				case STRING:
				case ESCAPE:
					put(ch);
					break;

				default:
					throw syntax("expecting-escape", ch);
				}
				break;

			case EOL:
				throw syntax("unterminated-string", str(started));

			default:
				if (ch < 32) {
					throw problem("embedded-control");
				}
				put(ch);
			}
		}
	}


	private void trim() {
		int i = buffer.length() - 1;
		while (i >= 0) {
			if (! white(buffer.charAt(i))) {
				buffer.setLength(i + 1);
				return;
			}
            // The char at i is whitespace; back toward beginning - to find non-whitespace.
            i = i - 1;
		}
		buffer.setLength(0);
	}


	private void seek() {
		while(true) {
			char ch = read();
			if ((ch == EOL) || (ch == STRING) || ! white(ch)) {
				pushback(ch);
				return;
			}
		}
	}


	/**
	 * Parse out a character literal. A character literal can be in multiple
	 * forms, included:
	 * 		Unicode:	'U+0045'
	 * 		Keyword:	'space', 'cr', 'NL'
	 * 		Simple:		'A'
	 * 		Classic:	'\n'
	 * All of these forms are auto-detected and converted into the actual
	 * character before creating the token.
	 */
	private Token character() {
		char ch = read();
		switch (ch) {
		case 'U':
			char n = read();
			switch(n) {
			case CHAR:
				return new Literal(this, Species.CHAR, ch);

			case '+':
				for (int i = 0; i < 4; i ++) {
					put(hex());
				}
				expect(CHAR);
				return new Literal(this, Species.CHAR,
				                   new Character((char)Integer.parseInt(buf(), 16)));

			case EOL:
				throw eof();

			default:
				put(ch);
				put(n);
				return charword();
			}

		case ESCAPE:
			ch = read();
			switch(ch) {
			case ESCAPE:
			case '\'':
				break;

			case 'r':
				ch = '\r';
				break;

			case 'n':
				ch = '\n';
				break;

			default:
				throw syntax("expecting-escape", ch);
			}
			expect(CHAR);
			return new Literal(this, Species.CHAR, ch);

		case EOL:
			throw eof();

		default:
			char x = read();
			switch(x) {
			case CHAR:
				return new Literal(this, Species.CHAR, ch);

			case EOL:
				throw eof();

			default:
				put(ch);
				put(x);
				return charword();
			}
		}
	}


	private Token charword() {
		while(true) {
			char ch = read();
			switch(ch) {
			case CHAR:
				Character c = charwords.get(buf().toLowerCase(LOC));
				if (c == null) {
					throw syntax("invalid-char-keyword", quote(buffer));
				}
				return new Literal(this, Species.CHAR, c);

			case EOL:
				throw eof();

			default:
				if (letter(ch) || digit(ch)) {
					put(ch);
				} else {
					throw syntax("invalid-keyword-char", ch);
				}
			}
		}

	}


	/**
	 * An escaped identifier, such as `shopping cart` or `last order.date`.
	 */
	private Token escaped() {
		while(true) {
			char ch = read();
			if (ch == IDENT) {
				return new Identifier(this, buf());
			}
			if (ch == EOL) {
				throw eof();
			}
			if (ch < 32) {
				throw syntax("invalid-escaped-ident", ch);
			}
			put(ch);
		}
	}


	/**
	 * A leading dash can indicate a negative number or the start of an operator.
	 */
	private Token dash() {
		char ch = read();
		if (in(currencies, ch)) {
			put(DASH);
			return money(ch);
		}
		if (ch == '0') {
			put(DASH);
			put(ch);
			ch = read();
			if (ch != '.') {
				throw syntax("expecting-numeric", ch);
			}
			put(ch);
			return fractional();
		}
		if (digit(ch)) {
			put('-');
			put(ch);
			return number();
		}
		pushback(ch);
		put(DASH);
		return operator();
	}


	/**
	 * A zero can indicate any of the following:
	 * 	- Just zero
	 * 	- A hexadecimal number in the form 0xF3BC
	 * 	- A decimal value, such as 0.03
	 * 	- An improper fraction, such as 0/3
	 */
	private Token zero() {
		char ch = read();
		switch (ch) {
		case DASH:
			// extended literal, such as 0-seconds
			return extend(new Literal(this, Species.INTEGER, ZERO));

		case '.':
			// Something like 0.450
			put("0.");
			return fractional();

		case PERCENT:
			// 0%, which is useless, but valid.
			return new Literal(this, Species.PERCENT,
			                   new Percent(Int.zero));

		case 'x':
			// A hexadecimal number, such as 0x4BC32 or 0x00FF,FF32
			return hexadecimal();

		case '/':
			// fraction with zero as the numerator, such as 0/3
			return extend(
			           new Literal(this, Species.FRACTION,
			                       new Fraction(ZERO, smallint(integer()))));

		case ':':
			// something like 0:45 in 24-hour time format
			return time(0);

		case EOL:
			return new Literal(this, Species.INTEGER, ZERO);

		default:
			if (letter(ch)) {
				pushback(ch);
				return extend(new Literal(this, Species.INTEGER, ZERO));
			}
			if (terminal(ch)) {
				pushback(ch);
				return new Literal(this, Species.INTEGER, ZERO);
			}
			if (digit(ch)) {
				/*
				 * In this case, we have a time value specified with a
				 * leading '0' character. For example: "03:15pm".
				 */
				char c = read();
				if (c == ':') {
					return time(ival(Character.toString(ch)));
				}
				throw problem("invalid-time-literal");
			}
			if (white(ch)) {
				return new Literal(this, Species.INTEGER, ZERO);
			}
			throw syntax("syntax-near", ch);
		}
	}


	private Token hexadecimal() {
		boolean end = false;
		while(! end) {
			char ch = read();
			if (ch == COMMA) {
				ch = read();
				if (hex(ch)) {
					put(ch);
				} else {
					if (terminal(ch)) {
						pushback(ch);
						pushback(COMMA);
						end = true;
					} else {
						throw syntax("syntax-near", ch);
					}
				}
			} else {
				if (hex(ch)) {
					put(ch);
				} else {
					if (terminal(ch)) {
						pushback(ch);
						end = true;
					} else {
						throw syntax("syntax-near", ch);
					}
				}
			}
		}
		return new Literal(this, Species.INTEGER, new BigInteger(buf(), 16));
	}


	/**
	 * Numbers are, well, complicated. Encountering a numeric digit may mean
	 * we are seeing a date, a time, fractions, percentages, etc.
	 */
	private Token number() {
		char prev = EOL;
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
			} else {
				switch(ch) {
				case COMMA:
					/*
					 * We ignore any commas that are contained in the whole
					 * portion of a number. We only detect two commas in a
					 * row as an error.
					 */
					if (prev == COMMA) {
						put(ch);
						throw syntax("too-many-commas", buf());
					}
					char peek = read();
					if (digit(peek)) {
						put(peek);
					} else {
						if (white(peek) || terminal(peek)) {
							/**
							 * In this case, the comma was followed by
							 * whitespace, so its simply an integer argument
							 * to a function. We put that syntax back so we
							 * see it again elsewhere.
							 */
							pushback(peek);
							pushback(COMMA);
							return new Literal(this, Species.INTEGER, intval());
						}
						throw syntax("expecting-numeric", ch);
					}
					break;

				case PERIOD:
					if (prev == COMMA) {
						throw syntax("expecting-numeric", quote('.'));
					}
					put(ch);
					return fractional();

				case COLON:
					if (config.dates()) {
						String time = buf();
						int hours = ival(time);
						if ((hours < 0) || (hours > 23)) {
							throw syntax("invalid-hours", time);
						}
						return time(hours);
					}
					throw syntax("syntax-near", ch);

				case PERCENT:
					return new Literal(this, Species.PERCENT,
						new Percent(Int.from(smallint(intval())).divide(Int.from(100))));

				case '/':
					if (config.dates()) {
						return fracdate(ival(buf()));
					}
					return extend(new Literal(this, Species.FRACTION,
					                          new Fraction(ival(buf()), ival(digits()))));

				case '-':
					/*
					 * IAt this point, the dash can indicate a mixed fraction,
					 * such as 3-1/2mm, or an extended literal, such as
					 * 45in. If the next character is a digit, its a
					 * fraction.
					 */
					ch = read();
					if (digit(ch)) {
						pushback(ch);
						return mixed(intval());
					} else {
						// extended literal
						pushback(ch);
						pushback('-');
						return extend(new Literal(this, Species.INTEGER, intval()));
					}

				default:
					Literal n = new Literal(this, Species.INTEGER, intval());
					if (letter(ch)) {
						pushback(ch);
						return extend(n);
					}
					pushback(ch);
					return n;
				}
			}
			prev = ch;
		}
	}


	/**
	 * At this point, we have already consumed the whole portion and the decimal
	 * point, and all we need to finish is the fractional portion.
	 */
	private Literal fractional() {
		int i = 0;
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
				i++;
			} else {
				if (i == 0) {
					throw syntax("expecting-numeric", ch);
				}
				pushback(ch);
				return extend(new Literal(this, Species.DECIMAL,
				                          new BigDecimal(buf())));
			}
		}
	}


	/**
	 * A mixed number, such as 3-1/2. At this point, we will have consumed the
	 * whole number portion and the "-" character.
	 */
	private Token mixed(Number whole) {
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
			} else {
				if (ch == '/') {
					Number numerator = intval();
					return extend(new Literal(this, Species.FRACTION,
					                          Fraction.from(smallint(whole), smallint(numerator),
					                                        smallint(integer()))));
				} else {
					throw problem("invalid-fraction",
					              nv("fraction", whole.toString() + '-' + buf()));
				}
			}
		}
	}


	/**
	 * At this point, we have an integer value and have encountered the '/'
	 * character. So, this can be a date literal (such as 2009/12/15) or a
	 * fraction (such as 3/4). We have to dig in further to figure it out.
	 */
	private Token fracdate(int value) {
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
			} else {
				if (ch == '/') {
					/*
					 * We have encountered a second '/', so it must be a date.
					 * We leave validation and parsing of the day to the next
					 * function.
					 */
					return date(value, ival(buf()));
				} else {
					pushback(ch);
					return extend(new Literal(this, Species.FRACTION,
					                          new Fraction(value, smallint(intval()))));
				}
			}
		}
	}


	/**
	 * Process the remainder of the date. At this point, we have consumed
	 * the first two '/' characters. All that is left is to pick up the day
	 * and validate the entire thing.
	 */
	private Token date(int year, int month) {
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
			} else {
				pushback(ch);
				int day = ival(buf());
				if (! Date.valid(month, day, year)) {
					throw syntax("invalid-date-literal",
					             (year + "/" + month + '/' + day));
				}
				return new Literal(this, Species.DATE, new Date(month, day, year));
			}
		}
	}


	/**
	 * At this point, we have read a single letter. That means it can be an
	 * identifier, a keyword (such as "true"), or one of the keyword operators
	 * (such as "or"). We disambiguate them here.
	 */
	private Token resolve() {
		while(true) {
			char ch = read();
			if (ident(ch)) {
				put(ch);
			} else {
				pushback(ch);
				String s = buf();
				Keyword k = keywords.get(s);
				if (k != null) {
					return k.construct(this);
				}
				OperatorDefinition opdef = config.operator(s);
				if (opdef != null) {
					if (opdef.symbol().equals("as")) {
						/*
						 * To support the odd bit that is the "as" operator,
						 * we need to have a distinctly separate mode in the
						 * lexer.
						 */
						as = true;
					}
					return new Op(this, opdef);
				}
				return new Identifier(this, s);
			}
		}
	}


	/**
	 * At this point, we already have at least one operator character in the
	 * buffer. We process until we find a terminal.
	 */
	private Token operator() {
		while(true) {
			char ch = read();
			if (in(operators, ch)) {
				put(ch);
			} else {
				pushback(ch);
				String symbol = buf();
				OperatorDefinition def = config.operator(symbol);
				if (def == null) {
					throw syntax("invalid-operator", quote(symbol));
				}
				return new Op(this, def);
			}
		}
	}


	// $250,500.00
	// $45,000-usd
	// $45,000JPY
	private Token money(char currency) {
		Token value = number();
		if (value instanceof Literal) {
			Literal v = new Literal(this, Species.MONEY, ((Literal)value).decimal());
			return v.prefix(currency);
		}
		throw syntax("not-money", value.toString());
	}


	/**
	 * At this point, we have consumed the hours and the colon. The remainder
	 * indicates if we are in 24-hour time or in 12-hour (AM/PM) time.
	 */
	private Token time(int hours) {
		put(digit());
		put(digit());
		int minutes = ival(buf());
		if ((minutes < 0) || (minutes > 59)) {
			throw problem("invalid-time");
		}
		if ((hours > 0) && (hours < 13)) {
			// ambiguous, must check next letter
			char ch = read();
			if (terminal(ch)) {
				return new Literal(this, Species.TIME, new Time(hours, minutes));
			} else {
				char m = Character.toUpperCase(ch);
				if ((m == 'A') || (m == 'P')) {
					ch = Character.toUpperCase(read());
					if (ch != 'M') {
						throw problem("invalid-time");
					}
					ch = read();
					if (! terminal(ch)) {
						throw syntax("syntax-near", ch);
					}
					pushback(ch);
					return new Literal(this, Species.TIME, new Time(hours,
					                   minutes, m));
				}
				throw problem("invalid-time");
			}
		}
		return new Literal(this, Species.TIME, new Time(hours, minutes));
	}


	/**
	 * Take a literal and add the suffix to it if it is an extended literal.
	 * For example 32mm or 3.5in. We also deal with the damn percent here, too.
	 */
	private Literal extend(Literal literal) {
		char ch = read();
		switch(ch) {
		case DASH:
			return literal.suffix(suffix());

		case PERCENT:
			if (literal.percent()) {
				return literal.percentify();
			}
			throw problem("non-numeric-percent");

		default:
			if (letter(ch)) {
				put(ch);
				return literal.suffix(suffix());
			}
			pushback(ch);
			return literal;
		}
	}


	private String suffix() {
		while(true) {
			char ch = read();
			if (letter(ch) || digit(ch) || (ch == '/')) {
				put(ch);
			} else {
				pushback(ch);
				return buf();
			}
		}
	}


	private void put(String chars) {
		buffer.append(chars);
	}


	private void put(char ch) {
		buffer.append(ch);
	}


	private void skip() {
		while(true) {
			char ch = read();
			if (ch == EOL) {
				return;
			}
			if (! white(ch)) {
				pushback(ch);
				return;
			}
		}
	}


	/**
	 * Discard the remainder of the line until either the terminal newline is
	 * reached or the end of the input is reached.
	 */
	private void discard() {
		while(true) {
			char ch = read();
			if ((ch == '\n') || (ch == EOL)) {
				return;
			}
		}
	}


	private char read() {
		if (pushed != null) {
			char x = pushed.value();
			pushed = pushed.pop();
			return x;
		}
		try {
			if (input == null) {
				return EOL;
			}
			int i = input.read();
			if (i < 0) {
				line ++;
				close();
				return EOL;
			}
			char ch = (char)i;
			if (ch == '\n') {
				line ++;
			}
			if (ch > 127) {
				Character sub = alt.get(ch);
				if (sub != null) {
					ch = sub;
				}
			}
			return ch;
		} catch (IOException e) {
			close();
			throw abort(e);
		}
	}


	private void pushback(char ch) {
		pushed = new Pushback(pushed, ch);
	}


	private boolean white(char ch) {
		return ch <= 32;
	}


	private boolean digit(char ch) {
		return Character.isDigit(ch);
	}


	private boolean letter(char ch) {
		return Character.isLetter(ch);
	}


	private boolean ident(char ch) {
		return digit(ch) || letter(ch) || in(".?-_!", ch);
	}


	private boolean terminal(char ch) {
		return (ch == EOL) || (ch == COMMA) || (ch == CLOSE);
	}


	private boolean in(String chars, char ch) {
		return chars.indexOf(ch) >= 0;
	}



	private CompilationException syntax(String key, char ch) {
		return problem(key, nv("that", ch));
	}


	private CompilationException syntax(String key, Object value) {
		return problem(key, nv("that", value));
	}


	private CompilationException problem(String key, NameValue... values) {
		close();
		return new CompilationException(this, key, values);
	}


	private void expect(char expected) {
		char actual = read();
		if (actual == EOL) {
			throw problem("unexpected-eof");
		}
		if (actual != expected) {
			throw problem("unexpected-char",
			              nv("expected", quote(expected)),
			              nv("actual", quote(actual)));
		}
	}


	private String buf() {
		String b = buffer.toString();
		reset();
		return b;
	}


	private char digit() {
		char ch = read();
		if (digit(ch)) {
			return ch;
		}
		throw syntax("expecting-numeric", nv("actual", ch));
	}


	private String digits() {
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
			} else {
				pushback(ch);
				String b = buf();
				if (b.length() == 0) {
					throw syntax("expecting-numeric", nv("actual", ch));
				}
				return b;
			}
		}
	}


	private Number integer() {
		while(true) {
			char ch = read();
			if (digit(ch)) {
				put(ch);
			} else {
				String b = buf();
				if (b.length() == 0) {
					throw syntax("expecting-numeric", nv("actual", ch));
				}
				pushback(ch);
				return b.length() < 10 ? Integer.valueOf(b) : new BigInteger(b);
			}
		}
	}


	private Number intval() {
		String b = buf();
		return b.length() < 10 ? Integer.valueOf(b) : new BigInteger(b);
	}


	private int ival(String b) {
		try {
			return Integer.parseInt(b);
		} catch (NumberFormatException e) {
			throw syntax("invalid-integer", b);
		}
	}


	private int smallint(Number n) {
		if (n instanceof Integer) {
			return (Integer)n;
		}
		throw syntax("invalid-integer", str(n));
	}


	private void reset() {
		buffer.setLength(0);
	}


	private char hex() {
		char ch = read();
		if (ch == EOL) {
			throw eof();
		}
		if (hex(ch)) {
			return ch;
		}
		throw syntax("expecting-hex-digit", ch);
	}


	private boolean hex(char ch) {
		char d = Character.toUpperCase(ch);
		return ((d >= '0') && (d <= '9')) || ((d >= 'A') && (d <= 'F'));
	}


	private CompilationException eof() {
		return problem("unexpected-eof");
	}


	private void close() {
		if (input != null) {
			try {
				input.close();
				input = null;
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		}
	}
}
